# coding:utf-8
import requests
import sys
import os
import time

# 这个文件是用来存储已经检查过且正常访问的url，避免重复检查，
# 以便多次运行脚本时候跳过已检查项目
known_valid_urls_file = 'known_valid_urls.txt'


def load_known_valid_urls_file():
    if os.path.isfile(known_valid_urls_file):
        return read_file_lines(known_valid_urls_file)
    else:
        return []


def read_file_lines(file):
    with open(file, 'r', encoding='utf8', errors='ignore') as f:
        return [l.strip() for l in f.readlines() if len(l) > 5]


def check_url(url):
    try:
        resp = requests.get(url, timeout=5)
        if (resp.status_code == 200):
            return True
        else:
            return False
    except Exception as e:
        print(e)
        return False


def append_line(line, file):
    with open(file, "a", encoding='utf-8') as fw:
        fw.write(line + os.linesep)


def write_file_lines(lines, file):
    with open(file, "w", encoding='utf-8') as fw:
        for line in lines:
            fw.write(line)


# Print iterations progress
def printProgressBar(iteration, total, prefix='', suffix='', decimals=1, length=100, fill='█', printEnd=" "):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=printEnd)
    # Print New Line on Complete
    if iteration == total:
        print()


if __name__ == "__main__":
    # urls_txt = r"D:\dl\样本数据.txt"
    urls_txt = r"D:\dl\rw.txt"
    urls_checked_txt = urls_txt.replace('.txt', 'checked.txt')
    known_valid_urls = set(load_known_valid_urls_file())
    input_lines = read_file_lines(urls_txt)
    all_urls_count = len(input_lines)
    print(f'{urls_txt} 包括 {all_urls_count} 行url')
    open(urls_checked_txt, 'w', encoding='utf8', errors='ignore').close()
    tried = 0
    printProgressBar(0, all_urls_count, prefix='Progress:', suffix='Complete', length=50)
    for url in input_lines:
        tried += 1
        printProgressBar(tried, all_urls_count, prefix='Progress:', suffix='Complete', length=50)
        if url in known_valid_urls:
            append_line(url, urls_checked_txt)
            continue
        if check_url(url):
            known_valid_urls.add(url)
            append_line(url, urls_checked_txt)
            append_line(url, known_valid_urls_file)
        else:
            append_line(url + '\t---无法访问', urls_checked_txt)
            print(url)
    print('end of main.')
